{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as np\n",
    "import numpy as np\n",
    "import scipy.sparse as ss\n",
    "import scipy.io as sio\n",
    "\n",
    "import cPickle\n",
    "import hashlib\n",
    "\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "import scipy.spatial.distance as ssd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of records:3137972\n"
     ]
    }
   ],
   "source": [
    "lines = 0\n",
    "fin = open(\"events.csv\",'rb')\n",
    "fin.readline()\n",
    "for line in fin:\n",
    "    cols = line.strip().split(\",\")\n",
    "    lines += 1\n",
    "fin.close()\n",
    "print(\"number of records:%d\" % lines)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of events in train & test :13418\n"
     ]
    }
   ],
   "source": [
    "eventIndex = cPickle.load(open(\"PE_eventIndex.pkl\",'rb'))\n",
    "n_events = len(eventIndex)\n",
    "print(\"number of events in train & test :%d\" % n_events)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'FeatureEng' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m\u001b[0m",
      "\u001b[1;31mNameError\u001b[0mTraceback (most recent call last)",
      "\u001b[1;32m<ipython-input-17-596150681fb7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnormalize\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[1;32mclass\u001b[0m \u001b[0mFeatureEng\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m     \u001b[0mFE\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mFeatureEng\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[0mfin\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"events.csv\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'rb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-17-596150681fb7>\u001b[0m in \u001b[0;36mFeatureEng\u001b[1;34m()\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mFeatureEng\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m     \u001b[0mFE\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mFeatureEng\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m \u001b[0mfin\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"events.csv\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'rb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[0mfin\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'FeatureEng' is not defined"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "import hashlib\n",
    "import locale\n",
    "\n",
    "from collections import defaultdict\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "class FeatureEng:\n",
    "    FE = FeatureEng()\n",
    "fin = open(\"events.csv\",'rb')\n",
    "fin.readline()\n",
    "eventPropMatrix = ss.dok_matrix((n_events,7))\n",
    "eventContMatrix = ss.dok_matrix((n_events,101))\n",
    "for line in fin.readlines():\n",
    "    cols = line.strip().split(\",\")\n",
    "    eventId = str(cols[0])\n",
    "    if eventIndex.has_key(eventId):\n",
    "        i = eventIndex[eventId]\n",
    "        \n",
    "        eventPropMatrix[i,0] = FE.getJionedYearMonth(cols[2])\n",
    "        eventPropMatrix[i,1] = FE.getFeatureHash(cols[3])\n",
    "        eventPropMatrix[i,2] = FE.getFeatureHash(cols[4])\n",
    "        eventPropMatrix[i,3] = FE.getFeatureHash(cols[5])\n",
    "        eventPropMatrix[i,4] = FE.getFeatureHash(cols[6])\n",
    "        eventPropMatrix[i,5] = FE.getFloatValue(cols[7])\n",
    "        eventPropMatrix[i,6] = FE.getFloatValue(cols[8])\n",
    "        \n",
    "        for j in rang(9,110):\n",
    "            eventContMatrix[i,j-9] = cols[j]\n",
    "fin.close()\n",
    "eventPropMatrix = normalize(eventPropMatrix,norm=\"12\",axis=0,copy=False)\n",
    "sio.mmwrite(\"EV_eventPropMatrix\",eventPropMatrix)\n",
    "eventContMatrix = normalize(eventContMatrix,norm=\"12\",axis=0,copy=False)\n",
    "sio.mmwrite(\"EV_eventContMatrix\",eventContMatrix)\n",
    "\n",
    "eventPropSim = ss.dok_matrix((n_events,n_events))\n",
    "eventContSim = ss.dok_matrix((n_events,n_events))\n",
    "uniqueEventPairs = cPickle.load(open(\"PE_uniqueEventPairs.pkl\",'rb'))\n",
    "\n",
    "for e1, e2 in uniqueEventPairs:\n",
    "    i=e1\n",
    "    j=e2\n",
    "    \n",
    "    if not eventPropSim.has_key((i,j)):\n",
    "        epsim = ssd.correlation(eventPropMatrix.getrow(i).todense(),eventPropMatrix.getrow(j).todense())\n",
    "        \n",
    "        eventPropSim[i, j] = epsim\n",
    "        eventPropSim[j, i] = epsim\n",
    "sio.mmwrite(\"EV_eventPropsim\",eventPropSim)\n",
    "sio.mmwrite(\"EV_eventPropsim\",eventContSim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eventPropsim.getrow(0).todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
